{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error occurred during loading data. Trying to use cache server https://fake-useragent.herokuapp.com/browsers/0.1.11\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 1317, in do_open\n",
      "    encode_chunked=req.has_header('Transfer-encoding'))\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 1252, in request\n",
      "    self._send_request(method, url, body, headers, encode_chunked)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 1298, in _send_request\n",
      "    self.endheaders(body, encode_chunked=encode_chunked)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 1247, in endheaders\n",
      "    self._send_output(message_body, encode_chunked=encode_chunked)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 1026, in _send_output\n",
      "    self.send(msg)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 966, in send\n",
      "    self.connect()\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py\", line 938, in connect\n",
      "    (self.host,self.port), self.timeout, self.source_address)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/socket.py\", line 727, in create_connection\n",
      "    raise err\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/socket.py\", line 716, in create_connection\n",
      "    sock.connect(sa)\n",
      "socket.timeout: timed out\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/fake_useragent/utils.py\", line 67, in get\n",
      "    context=context,\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 222, in urlopen\n",
      "    return opener.open(url, data, timeout)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 525, in open\n",
      "    response = self._open(req, data)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 543, in _open\n",
      "    '_open', req)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 503, in _call_chain\n",
      "    result = func(*args)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 1345, in http_open\n",
      "    return self.do_open(http.client.HTTPConnection, req)\n",
      "  File \"/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py\", line 1319, in do_open\n",
      "    raise URLError(err)\n",
      "urllib.error.URLError: <urlopen error timed out>\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/fake_useragent/utils.py\", line 166, in load\n",
      "    verify_ssl=verify_ssl,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/fake_useragent/utils.py\", line 122, in get_browser_versions\n",
      "    verify_ssl=verify_ssl,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/fake_useragent/utils.py\", line 84, in get\n",
      "    raise FakeUserAgentError('Maximum amount of retries reached')\n",
      "fake_useragent.errors.FakeUserAgentError: Maximum amount of retries reached\n"
     ]
    }
   ],
   "source": [
    "from fake_useragent import UserAgent\n",
    "import requests\n",
    "\n",
    "ua = UserAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from newspaper import Article\n",
    "from queue import Queue\n",
    "from urllib.parse import quote\n",
    "from unidecode import unidecode\n",
    "\n",
    "def get_date(load):\n",
    "    try:\n",
    "        date = re.findall(\n",
    "            '[-+]?[.]?[\\d]+(?:,\\d\\d\\d)*[\\.]?\\d*(?:[eE][-+]?\\d+)?', load\n",
    "        )\n",
    "        return '%s-%s-%s' % (date[2], date[0], date[1])\n",
    "    except Exce:\n",
    "        return False\n",
    "\n",
    "\n",
    "def run_parallel_in_threads(target, args_list):\n",
    "    globalparas = []\n",
    "    result = Queue()\n",
    "\n",
    "    def task_wrapper(*args):\n",
    "        result.put(target(*args))\n",
    "\n",
    "    threads = [\n",
    "        threading.Thread(target = task_wrapper, args = args)\n",
    "        for args in args_list\n",
    "    ]\n",
    "    for t in threads:\n",
    "        t.start()\n",
    "    for t in threads:\n",
    "        t.join()\n",
    "    while not result.empty():\n",
    "        globalparas.append(result.get())\n",
    "    globalparas = list(filter(None, globalparas))\n",
    "    return globalparas\n",
    "\n",
    "\n",
    "def get_article(link, news, date):\n",
    "    article = Article(link)\n",
    "    article.download()\n",
    "    article.parse()\n",
    "    article.nlp()\n",
    "    lang = 'ENGLISH'\n",
    "    if len(article.title) < 5 or len(article.text) < 5:\n",
    "        lang = 'INDONESIA'\n",
    "        print('found BM/ID article')\n",
    "        article = Article(link, language = 'id')\n",
    "        article.download()\n",
    "        article.parse()\n",
    "        article.nlp()\n",
    "    return {\n",
    "        'title': article.title,\n",
    "        'url': link,\n",
    "        'authors': article.authors,\n",
    "        'top-image': article.top_image,\n",
    "        'text': article.text,\n",
    "        'keyword': article.keywords,\n",
    "        'summary': article.summary,\n",
    "        'news': news,\n",
    "        'date': date,\n",
    "        'language': lang,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT = 0\n",
    "\n",
    "GOOGLE_NEWS_URL = 'https://www.google.com.my/search?q={}&source=lnt&tbs=cdr%3A1%2Ccd_min%3A{}%2Ccd_max%3A{}&tbm=nws&start={}'\n",
    "\n",
    "def forge_url(q, start, year_start, year_end):\n",
    "    global NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT\n",
    "    NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT += 1\n",
    "    return GOOGLE_NEWS_URL.format(\n",
    "        q.replace(' ', '+'), str(year_start), str(year_end), start\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.google.com.my/search?q=america&source=lnt&tbs=cdr%3A1%2Ccd_min%3A2005%2Ccd_max%3A2021&tbm=nws&start=0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_articles_index = 0\n",
    "url = forge_url('america', num_articles_index, '2005', '2021')\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}\n",
    "headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url, headers = headers, timeout = 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.content, 'html.parser')\n",
    "style=\"text-decoration:none;display:block\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'America Magazine'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('div', {'class': 'XTjFC WF4CUc'})[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[datetime.datetime(2010, 1, 2, 0, 0),\n",
       " datetime.datetime(2020, 7, 4, 16, 59, 12, 389201),\n",
       " datetime.datetime(2020, 7, 11, 16, 59, 12, 390981),\n",
       " datetime.datetime(2020, 6, 27, 16, 59, 12, 392462),\n",
       " datetime.datetime(2015, 9, 22, 0, 0),\n",
       " datetime.datetime(2020, 7, 4, 16, 59, 12, 397885),\n",
       " datetime.datetime(2020, 6, 18, 16, 59, 12, 400063),\n",
       " datetime.datetime(2019, 4, 9, 0, 0),\n",
       " datetime.datetime(2020, 5, 18, 0, 0),\n",
       " datetime.datetime(2020, 6, 18, 16, 59, 12, 409616)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[dateparser.parse(v.text) for v in soup.find_all('span', {'class': 'WG9SHc'})]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2020-07-04 16:59:46.017554'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dateparser\n",
    "\n",
    "str(dateparser.parse('2 weeks ago'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime, timedelta\n",
    "from dateutil import parser\n",
    "import dateparser\n",
    "\n",
    "def extract_links(content):\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    # return soup\n",
    "    today = datetime.now().strftime('%m/%d/%Y')\n",
    "    links_list = [v.attrs['href'] for v in soup.find_all('a', {'style': 'text-decoration:none;display:block'})]\n",
    "    dates_list = [v.text for v in soup.find_all('span', {'class': 'WG9SHc'})]\n",
    "    sources_list = [v.text for v in soup.find_all('div', {'class': 'XTjFC WF4CUc'})]\n",
    "    output = []\n",
    "    for (link, date, source) in zip(links_list, dates_list, sources_list):\n",
    "        try:\n",
    "            date = str(dateparser.parse(date))\n",
    "        except:\n",
    "            pass\n",
    "        output.append((link, date, source))\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('https://www.americamagazine.org/',\n",
       "  '2010-01-02 00:00:00',\n",
       "  'America Magazine'),\n",
       " ('https://www.theatlantic.com/ideas/archive/2020/07/america-land-pathetic/613747/',\n",
       "  '2020-07-04 17:04:57.632971',\n",
       "  'The Atlantic'),\n",
       " ('https://www.nytimes.com/2020/07/07/world/europe/american-passport-privilege-coronavirus.html',\n",
       "  '2020-07-11 17:04:57.634208',\n",
       "  'The New York Times'),\n",
       " ('https://www.theatlantic.com/international/archive/2020/06/america-image-power-trump/613228/',\n",
       "  '2020-06-27 17:04:57.635410',\n",
       "  'The Atlantic'),\n",
       " ('https://www.bbc.com/news/world-us-canada-34331761',\n",
       "  '2015-09-22 00:00:00',\n",
       "  'BBC News'),\n",
       " ('https://www.theatlantic.com/ideas/archive/2020/07/when-trumps-understanding-of-the-world-is-flipped-on-its-head/613840/',\n",
       "  '2020-07-04 17:04:57.639417',\n",
       "  'The Atlantic'),\n",
       " ('https://www.theatlantic.com/ideas/archive/2020/06/why-americas-institutions-are-failing/613078/',\n",
       "  '2020-06-18 17:04:57.640774',\n",
       "  'The Atlantic'),\n",
       " ('https://www.pewsocialtrends.org/2019/04/09/race-in-america-2019/',\n",
       "  '2019-04-09 00:00:00',\n",
       "  \"Pew Research Center's Social and Demographic Trends\"),\n",
       " ('https://www.brookings.edu/blog/the-avenue/2020/05/18/covid-19-shows-that-americas-broadband-plan-is-still-in-beta/',\n",
       "  '2020-05-18 00:00:00',\n",
       "  'Brookings Institution'),\n",
       " ('https://www.theatlantic.com/ideas/archive/2020/05/1968-and-2020-lessons-from-americas-worst-year-so-far/612415/',\n",
       "  '2020-06-18 17:04:57.647910',\n",
       "  'The Atlantic')]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = extract_links(response.content)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
